{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Copyright (C) 2016 - 2019 Pinard Liu(liujianping-ok@163.com)\n",
    "\n",
    "https://www.cnblogs.com/pinard\n",
    "\n",
    "Permission given to modify the code as long as you keep this declaration at the top\n",
    "\n",
    "scikit-learn 梯度提升树(GBDT)调参小结 https://www.cnblogs.com/pinard/p/6143927.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Python27\\lib\\site-packages\\sklearn\\cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n",
      "C:\\Python27\\lib\\site-packages\\sklearn\\grid_search.py:43: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn import cross_validation, metrics\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "\n",
    "import matplotlib.pylab as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    19680\n",
       "1      320\n",
       "Name: Disbursed, dtype: int64"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('train_modified.csv')\n",
    "target='Disbursed' # Disbursed的值就是二元分类的输出\n",
    "IDcol = 'ID'\n",
    "train['Disbursed'].value_counts() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_columns = [x for x in train.columns if x not in [target, IDcol]]\n",
    "X = train[x_columns]\n",
    "y = train['Disbursed']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy : 0.9852\n",
      "AUC Score (Train): 0.900531\n"
     ]
    }
   ],
   "source": [
    "gbm0 = GradientBoostingClassifier(random_state=10)\n",
    "gbm0.fit(X,y)\n",
    "y_pred = gbm0.predict(X)\n",
    "y_predprob = gbm0.predict_proba(X)[:,1]\n",
    "print \"Accuracy : %.4g\" % metrics.accuracy_score(y.values, y_pred)\n",
    "print \"AUC Score (Train): %f\" % metrics.roc_auc_score(y, y_predprob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([mean: 0.81285, std: 0.01967, params: {'n_estimators': 20},\n",
       "  mean: 0.81438, std: 0.01947, params: {'n_estimators': 30},\n",
       "  mean: 0.81451, std: 0.01933, params: {'n_estimators': 40},\n",
       "  mean: 0.81618, std: 0.01848, params: {'n_estimators': 50},\n",
       "  mean: 0.81751, std: 0.01736, params: {'n_estimators': 60},\n",
       "  mean: 0.81547, std: 0.01900, params: {'n_estimators': 70},\n",
       "  mean: 0.81299, std: 0.01860, params: {'n_estimators': 80}],\n",
       " {'n_estimators': 60},\n",
       " 0.8175146087398375)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_test1 = {'n_estimators':range(20,81,10)}\n",
    "gsearch1 = GridSearchCV(estimator = GradientBoostingClassifier(learning_rate=0.1, min_samples_split=300,\n",
    "                                  min_samples_leaf=20,max_depth=8,max_features='sqrt', subsample=0.8,random_state=10), \n",
    "                       param_grid = param_test1, scoring='roc_auc',iid=False,cv=5)\n",
    "gsearch1.fit(X,y)\n",
    "gsearch1.grid_scores_, gsearch1.best_params_, gsearch1.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([mean: 0.81199, std: 0.02073, params: {'min_samples_split': 100, 'max_depth': 3},\n",
       "  mean: 0.81267, std: 0.01985, params: {'min_samples_split': 300, 'max_depth': 3},\n",
       "  mean: 0.81238, std: 0.01937, params: {'min_samples_split': 500, 'max_depth': 3},\n",
       "  mean: 0.80925, std: 0.02051, params: {'min_samples_split': 700, 'max_depth': 3},\n",
       "  mean: 0.81846, std: 0.01843, params: {'min_samples_split': 100, 'max_depth': 5},\n",
       "  mean: 0.81630, std: 0.01810, params: {'min_samples_split': 300, 'max_depth': 5},\n",
       "  mean: 0.81315, std: 0.01898, params: {'min_samples_split': 500, 'max_depth': 5},\n",
       "  mean: 0.81262, std: 0.02090, params: {'min_samples_split': 700, 'max_depth': 5},\n",
       "  mean: 0.81807, std: 0.02004, params: {'min_samples_split': 100, 'max_depth': 7},\n",
       "  mean: 0.82137, std: 0.01733, params: {'min_samples_split': 300, 'max_depth': 7},\n",
       "  mean: 0.81703, std: 0.01773, params: {'min_samples_split': 500, 'max_depth': 7},\n",
       "  mean: 0.81383, std: 0.02327, params: {'min_samples_split': 700, 'max_depth': 7},\n",
       "  mean: 0.81107, std: 0.02178, params: {'min_samples_split': 100, 'max_depth': 9},\n",
       "  mean: 0.80944, std: 0.02612, params: {'min_samples_split': 300, 'max_depth': 9},\n",
       "  mean: 0.81476, std: 0.01973, params: {'min_samples_split': 500, 'max_depth': 9},\n",
       "  mean: 0.81601, std: 0.02576, params: {'min_samples_split': 700, 'max_depth': 9},\n",
       "  mean: 0.81091, std: 0.02227, params: {'min_samples_split': 100, 'max_depth': 11},\n",
       "  mean: 0.81309, std: 0.02696, params: {'min_samples_split': 300, 'max_depth': 11},\n",
       "  mean: 0.81713, std: 0.02379, params: {'min_samples_split': 500, 'max_depth': 11},\n",
       "  mean: 0.81347, std: 0.02702, params: {'min_samples_split': 700, 'max_depth': 11},\n",
       "  mean: 0.81444, std: 0.01813, params: {'min_samples_split': 100, 'max_depth': 13},\n",
       "  mean: 0.80825, std: 0.02291, params: {'min_samples_split': 300, 'max_depth': 13},\n",
       "  mean: 0.81923, std: 0.01693, params: {'min_samples_split': 500, 'max_depth': 13},\n",
       "  mean: 0.81382, std: 0.02258, params: {'min_samples_split': 700, 'max_depth': 13}],\n",
       " {'max_depth': 7, 'min_samples_split': 300},\n",
       " 0.8213724275914632)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_test2 = {'max_depth':range(3,14,2), 'min_samples_split':range(100,801,200)}\n",
    "gsearch2 = GridSearchCV(estimator = GradientBoostingClassifier(learning_rate=0.1, n_estimators=60, min_samples_leaf=20, \n",
    "      max_features='sqrt', subsample=0.8, random_state=10), \n",
    "   param_grid = param_test2, scoring='roc_auc',iid=False, cv=5)\n",
    "gsearch2.fit(X,y)\n",
    "gsearch2.grid_scores_, gsearch2.best_params_, gsearch2.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([mean: 0.81828, std: 0.02251, params: {'min_samples_split': 800, 'min_samples_leaf': 60},\n",
       "  mean: 0.81731, std: 0.02344, params: {'min_samples_split': 1000, 'min_samples_leaf': 60},\n",
       "  mean: 0.82220, std: 0.02250, params: {'min_samples_split': 1200, 'min_samples_leaf': 60},\n",
       "  mean: 0.81447, std: 0.02125, params: {'min_samples_split': 1400, 'min_samples_leaf': 60},\n",
       "  mean: 0.81495, std: 0.01626, params: {'min_samples_split': 1600, 'min_samples_leaf': 60},\n",
       "  mean: 0.81528, std: 0.02140, params: {'min_samples_split': 1800, 'min_samples_leaf': 60},\n",
       "  mean: 0.81590, std: 0.02517, params: {'min_samples_split': 800, 'min_samples_leaf': 70},\n",
       "  mean: 0.81573, std: 0.02207, params: {'min_samples_split': 1000, 'min_samples_leaf': 70},\n",
       "  mean: 0.82021, std: 0.02521, params: {'min_samples_split': 1200, 'min_samples_leaf': 70},\n",
       "  mean: 0.81512, std: 0.01995, params: {'min_samples_split': 1400, 'min_samples_leaf': 70},\n",
       "  mean: 0.81395, std: 0.02081, params: {'min_samples_split': 1600, 'min_samples_leaf': 70},\n",
       "  mean: 0.81587, std: 0.02082, params: {'min_samples_split': 1800, 'min_samples_leaf': 70},\n",
       "  mean: 0.82064, std: 0.02698, params: {'min_samples_split': 800, 'min_samples_leaf': 80},\n",
       "  mean: 0.81490, std: 0.02475, params: {'min_samples_split': 1000, 'min_samples_leaf': 80},\n",
       "  mean: 0.82009, std: 0.02568, params: {'min_samples_split': 1200, 'min_samples_leaf': 80},\n",
       "  mean: 0.81850, std: 0.02226, params: {'min_samples_split': 1400, 'min_samples_leaf': 80},\n",
       "  mean: 0.81855, std: 0.02099, params: {'min_samples_split': 1600, 'min_samples_leaf': 80},\n",
       "  mean: 0.81666, std: 0.02249, params: {'min_samples_split': 1800, 'min_samples_leaf': 80},\n",
       "  mean: 0.81960, std: 0.02437, params: {'min_samples_split': 800, 'min_samples_leaf': 90},\n",
       "  mean: 0.81560, std: 0.02235, params: {'min_samples_split': 1000, 'min_samples_leaf': 90},\n",
       "  mean: 0.81936, std: 0.02542, params: {'min_samples_split': 1200, 'min_samples_leaf': 90},\n",
       "  mean: 0.81362, std: 0.02254, params: {'min_samples_split': 1400, 'min_samples_leaf': 90},\n",
       "  mean: 0.81429, std: 0.02417, params: {'min_samples_split': 1600, 'min_samples_leaf': 90},\n",
       "  mean: 0.81299, std: 0.02262, params: {'min_samples_split': 1800, 'min_samples_leaf': 90},\n",
       "  mean: 0.82000, std: 0.02511, params: {'min_samples_split': 800, 'min_samples_leaf': 100},\n",
       "  mean: 0.82209, std: 0.01816, params: {'min_samples_split': 1000, 'min_samples_leaf': 100},\n",
       "  mean: 0.81821, std: 0.02337, params: {'min_samples_split': 1200, 'min_samples_leaf': 100},\n",
       "  mean: 0.81922, std: 0.02377, params: {'min_samples_split': 1400, 'min_samples_leaf': 100},\n",
       "  mean: 0.81545, std: 0.02221, params: {'min_samples_split': 1600, 'min_samples_leaf': 100},\n",
       "  mean: 0.81704, std: 0.02509, params: {'min_samples_split': 1800, 'min_samples_leaf': 100}],\n",
       " {'min_samples_leaf': 60, 'min_samples_split': 1200},\n",
       " 0.8222032996697154)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_test3 = {'min_samples_split':range(800,1900,200), 'min_samples_leaf':range(60,101,10)}\n",
    "gsearch3 = GridSearchCV(estimator = GradientBoostingClassifier(learning_rate=0.1, n_estimators=60,max_depth=7,\n",
    "                                     max_features='sqrt', subsample=0.8, random_state=10), \n",
    "                       param_grid = param_test3, scoring='roc_auc',iid=False, cv=5)\n",
    "gsearch3.fit(X,y)\n",
    "gsearch3.grid_scores_, gsearch3.best_params_, gsearch3.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy : 0.984\n",
      "AUC Score (Train): 0.908099\n"
     ]
    }
   ],
   "source": [
    "gbm1 = GradientBoostingClassifier(learning_rate=0.1, n_estimators=60,max_depth=7, min_samples_leaf =60, \n",
    "               min_samples_split =1200, max_features='sqrt', subsample=0.8, random_state=10)\n",
    "gbm1.fit(X,y)\n",
    "y_pred = gbm1.predict(X)\n",
    "y_predprob = gbm1.predict_proba(X)[:,1]\n",
    "print \"Accuracy : %.4g\" % metrics.accuracy_score(y.values, y_pred)\n",
    "print \"AUC Score (Train): %f\" % metrics.roc_auc_score(y, y_predprob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([mean: 0.82220, std: 0.02250, params: {'max_features': 7},\n",
       "  mean: 0.82241, std: 0.02421, params: {'max_features': 9},\n",
       "  mean: 0.82108, std: 0.02302, params: {'max_features': 11},\n",
       "  mean: 0.82064, std: 0.01900, params: {'max_features': 13},\n",
       "  mean: 0.82198, std: 0.01514, params: {'max_features': 15},\n",
       "  mean: 0.81355, std: 0.02053, params: {'max_features': 17},\n",
       "  mean: 0.81877, std: 0.01863, params: {'max_features': 19}],\n",
       " {'max_features': 9},\n",
       " 0.822412506351626)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_test4 = {'max_features':range(7,20,2)}\n",
    "gsearch4 = GridSearchCV(estimator = GradientBoostingClassifier(learning_rate=0.1, n_estimators=60,max_depth=7, min_samples_leaf =60, \n",
    "               min_samples_split =1200, subsample=0.8, random_state=10), \n",
    "                       param_grid = param_test4, scoring='roc_auc',iid=False, cv=5)\n",
    "gsearch4.fit(X,y)\n",
    "gsearch4.grid_scores_, gsearch4.best_params_, gsearch4.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([mean: 0.81828, std: 0.02392, params: {'subsample': 0.6},\n",
       "  mean: 0.82344, std: 0.02708, params: {'subsample': 0.7},\n",
       "  mean: 0.81673, std: 0.02196, params: {'subsample': 0.75},\n",
       "  mean: 0.82241, std: 0.02421, params: {'subsample': 0.8},\n",
       "  mean: 0.82285, std: 0.02446, params: {'subsample': 0.85},\n",
       "  mean: 0.81738, std: 0.02236, params: {'subsample': 0.9}],\n",
       " {'subsample': 0.7},\n",
       " 0.8234378969766262)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_test5 = {'subsample':[0.6,0.7,0.75,0.8,0.85,0.9]}\n",
    "gsearch5 = GridSearchCV(estimator = GradientBoostingClassifier(learning_rate=0.1, n_estimators=60,max_depth=7, min_samples_leaf =60, \n",
    "               min_samples_split =1200, max_features=9, random_state=10), \n",
    "                       param_grid = param_test5, scoring='roc_auc',iid=False, cv=5)\n",
    "gsearch5.fit(X,y)\n",
    "gsearch5.grid_scores_, gsearch5.best_params_, gsearch5.best_score_\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy : 0.984\n",
      "AUC Score (Train): 0.905324\n"
     ]
    }
   ],
   "source": [
    "gbm2 = GradientBoostingClassifier(learning_rate=0.05, n_estimators=120,max_depth=7, min_samples_leaf =60, \n",
    "               min_samples_split =1200, max_features=9, subsample=0.7, random_state=10)\n",
    "gbm2.fit(X,y)\n",
    "y_pred = gbm2.predict(X)\n",
    "y_predprob = gbm2.predict_proba(X)[:,1]\n",
    "print \"Accuracy : %.4g\" % metrics.accuracy_score(y.values, y_pred)\n",
    "print \"AUC Score (Train): %f\" % metrics.roc_auc_score(y, y_predprob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy : 0.984\n",
      "AUC Score (Train): 0.908581\n"
     ]
    }
   ],
   "source": [
    "gbm3 = GradientBoostingClassifier(learning_rate=0.01, n_estimators=600,max_depth=7, min_samples_leaf =60, \n",
    "               min_samples_split =1200, max_features=9, subsample=0.7, random_state=10)\n",
    "gbm3.fit(X,y)\n",
    "y_pred = gbm3.predict(X)\n",
    "y_predprob = gbm3.predict_proba(X)[:,1]\n",
    "print \"Accuracy : %.4g\" % metrics.accuracy_score(y.values, y_pred)\n",
    "print \"AUC Score (Train): %f\" % metrics.roc_auc_score(y, y_predprob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy : 0.984\n",
      "AUC Score (Train): 0.908232\n"
     ]
    }
   ],
   "source": [
    "gbm4 = GradientBoostingClassifier(learning_rate=0.005, n_estimators=1200,max_depth=7, min_samples_leaf =60, \n",
    "               min_samples_split =1200, max_features=9, subsample=0.7, random_state=10)\n",
    "gbm4.fit(X,y)\n",
    "y_pred = gbm4.predict(X)\n",
    "y_predprob = gbm4.predict_proba(X)[:,1]\n",
    "print \"Accuracy : %.4g\" % metrics.accuracy_score(y.values, y_pred)\n",
    "print \"AUC Score (Train): %f\" % metrics.roc_auc_score(y, y_predprob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy : 0.984\n",
      "AUC Score (Train): 0.899583\n"
     ]
    }
   ],
   "source": [
    "gbm11 = GradientBoostingClassifier(learning_rate=0.1, n_estimators=60,max_depth=7, min_samples_leaf =60, \n",
    "               min_samples_split =1200, max_features=9, subsample=0.7, random_state=10)\n",
    "gbm11.fit(X,y)\n",
    "y_pred = gbm11.predict(X)\n",
    "y_predprob = gbm11.predict_proba(X)[:,1]\n",
    "print \"Accuracy : %.4g\" % metrics.accuracy_score(y.values, y_pred)\n",
    "print \"AUC Score (Train): %f\" % metrics.roc_auc_score(y, y_predprob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
